{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "editable": false,
        "id": "-r-SJUrp6b_t"
      },
      "source": [
        "## Setting up"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "editable": false,
        "id": "bmDhM2646b_v",
        "papermill": {
          "duration": 164.985343,
          "end_time": "2024-09-25T13:11:54.926296",
          "exception": false,
          "start_time": "2024-09-25T13:09:09.940953",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!pip install unsloth\n",
        "# Also get the latest nightly Unsloth!\n",
        "!pip install --force-reinstall --no-cache-dir --no-deps git+https://github.com/unslothai/unsloth.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T06:15:28.692133Z",
          "iopub.status.busy": "2025-01-25T06:15:28.69189Z",
          "iopub.status.idle": "2025-01-25T06:15:34.082361Z",
          "shell.execute_reply": "2025-01-25T06:15:34.081539Z",
          "shell.execute_reply.started": "2025-01-25T06:15:28.692112Z"
        },
        "id": "G1UpWpAQ6b_w",
        "outputId": "6cf7bf5c-9c83-4d8f-bfa5-ea3c3c45f492",
        "papermill": {
          "duration": 31.301473,
          "end_time": "2024-09-25T13:12:26.233487",
          "exception": false,
          "start_time": "2024-09-25T13:11:54.932014",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "🦥 Unsloth: Will patch your computer to enable 2x faster free finetuning.\n",
            "🦥 Unsloth Zoo will now patch everything to make training faster!\n"
          ]
        }
      ],
      "source": [
        "from unsloth import FastLanguageModel\n",
        "import torch\n",
        "max_seq_length = 2048\n",
        "dtype = None\n",
        "load_in_4bit = True"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bui9WQqZ9FX2"
      },
      "source": [
        "使用本地环境变量"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T06:15:34.083808Z",
          "iopub.status.busy": "2025-01-25T06:15:34.083592Z",
          "iopub.status.idle": "2025-01-25T06:15:40.044666Z",
          "shell.execute_reply": "2025-01-25T06:15:40.044034Z",
          "shell.execute_reply.started": "2025-01-25T06:15:34.083789Z"
        },
        "id": "S6V8nb6Q6b_w",
        "papermill": {
          "duration": 0.618627,
          "end_time": "2024-09-25T13:12:26.85781",
          "exception": false,
          "start_time": "2024-09-25T13:12:26.239183",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [],
      "source": [
        "from huggingface_hub import login\n",
        "from kaggle_secrets import UserSecretsClient\n",
        "user_secrets = UserSecretsClient()\n",
        "\n",
        "hf_token = user_secrets.get_secret(\"HUGGINGFACE_TOKEN\")\n",
        "login(hf_token)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "wlLCW6ui9hNx"
      },
      "outputs": [],
      "source": [
        "## 使用colab环境变量\n",
        "from huggingface_hub import login\n",
        "from google.colab import userdata\n",
        "\n",
        "hf_token = userdata.get('deepseekr1')\n",
        "login(hf_token)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QMIlEq9dGDG0"
      },
      "source": [
        "可视化训练过程，将数据报送到wandb中"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T06:56:25.270546Z",
          "iopub.status.busy": "2025-01-25T06:56:25.270199Z",
          "iopub.status.idle": "2025-01-25T06:56:31.228898Z",
          "shell.execute_reply": "2025-01-25T06:56:31.228041Z",
          "shell.execute_reply.started": "2025-01-25T06:56:25.270519Z"
        },
        "id": "GqDYtZDK6b_w",
        "papermill": {
          "duration": 3.428482,
          "end_time": "2024-09-25T13:12:30.291772",
          "exception": false,
          "start_time": "2024-09-25T13:12:26.86329",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [],
      "source": [
        "import wandb\n",
        "\n",
        "wb_token = user_secrets.get_secret(\"wandb\")\n",
        "\n",
        "wandb.login(key=wb_token)\n",
        "run = wandb.init(\n",
        "    project='Fine-tune-DeepSeek-R1-Distill-Llama-8B on Medical COT Dataset',\n",
        "    job_type=\"training\",\n",
        "    anonymous=\"allow\"\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "editable": false,
        "id": "x3C_JY6e6b_x"
      },
      "source": [
        "## Loading the model and tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 284,
          "referenced_widgets": [
            "4afe538bb09d4cf698235e135640ab6e",
            "1fe6b27ff0cc4e9fa39b0b5840a25aec",
            "f811d4e961914901b3b458934512eb86",
            "9812fb88a8b3411d8675d238161dd7aa",
            "6dbb94366cc941a8ba336ba61ebf99ba",
            "1992bceec6b54ac2827285d39c8c12ee",
            "45f8a2d59bfe46008feebab8981d7146",
            "2e82c8498b854d41842977ef8f9bbb66",
            "adb31ccfc7ef415fb47a40ede28fe9db",
            "424e62fec0c0416ba13f5863391e08f9",
            "0a389bfe361f43578d55001f88a92fea",
            "82860a16b8d944ff86f854f215535379",
            "33edc00314bf444188376c01ea025038",
            "4e3945399c1642c1bdf724f8a01fcd02",
            "f58d5860eb854c698f6558e4422acced",
            "0f3a15d30fca4e5aa6321f1497e9c449",
            "e68571618bf748ebbf6d3e244e0e7f29",
            "e1d763ec083b49d0900cfe5d42d9b214",
            "972543b62d0a4d908101b7abc8c1ca15",
            "8e5e7c77e88a4d3b8b606f240b627a9d",
            "b8428a11dcfe4a2ea6d47b6e010fe3d2",
            "dca1f2da331542ae85818b544827dd96",
            "ff5beb41522f4b80bdb35c436f478bb6",
            "7ae90e66d3f1459ab5e0c202cf4228ab",
            "b01764e9ac74437aa174b132789417af",
            "029a61b64a0844dab51a9eb8ab949bf0",
            "9ab8e04684054331928edd757ce8dea3",
            "e5784f34cf184023ac60d15e4c347688",
            "3914a21f18114d6784bbb8f4867be344",
            "80b57123cbb64b2c9a27c5159a104556",
            "cf8df2dc586a4a9bba21066c50805fda",
            "4e17d9034b6d41c58bdc8ca94ea0c2fd",
            "ac02a01ce16e4fbca4ff1819a5153088",
            "e07fdea2d68c4284b43df47b7f94fceb",
            "b0dc37d789084b8e95ae90c155699ab7",
            "cf82cef940944506bbb654363a48cdb7",
            "fe6b626df24943978a0af261d29c7ee4",
            "724109739173448fb1965e01895f51a2",
            "41c36bc4fcb542a0bbc3443cd4c742ea",
            "08749a83ae4a47128e23b69dd791ab6e",
            "ba91815af2f84652987bd01420bb061f",
            "f500319ad888458f902f5fc4fc30d528",
            "44d3cf803d244775826d566c4d9ca859",
            "51ad33f2e8dc44f79a8e8fd9a86e22b5",
            "cdc489ca63484d4abb3ac39a3f93e33c",
            "30b0da59ccb241439ee988cf52bf81b3",
            "1cd217e8233d495db151f5debdf9f00b",
            "76bc4ff5adb746a69954699cf5465c2f",
            "bee5e75849cf448fa6c5eb2ef61f48d4",
            "1f2e54775e6e4288a2a772b75d104fbb",
            "e15cb3d0d72a475ea39eb1ef761dc827",
            "eb8becd866564ffd859e22bbc61a93e8",
            "7a50ddf657a04aef8c68c4100323fcea",
            "045a773300914b91a43f7f4922baf8cc",
            "9ef2064efb05485b8b63bb3db815a8ef"
          ]
        },
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T06:15:40.04576Z",
          "iopub.status.busy": "2025-01-25T06:15:40.045459Z",
          "iopub.status.idle": "2025-01-25T06:18:24.985511Z",
          "shell.execute_reply": "2025-01-25T06:18:24.984666Z",
          "shell.execute_reply.started": "2025-01-25T06:15:40.045729Z"
        },
        "id": "eS3RngLN6b_x",
        "outputId": "a83754d4-12db-49aa-f563-47287a9b5b72",
        "papermill": {
          "duration": 33.212874,
          "end_time": "2024-09-25T13:13:03.511347",
          "exception": false,
          "start_time": "2024-09-25T13:12:30.298473",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "==((====))==  Unsloth 2025.1.8: Fast Llama patching. Transformers: 4.47.1.\n",
            "   \\\\   /|    GPU: Tesla T4. Max memory: 14.741 GB. Platform: Linux.\n",
            "O^O/ \\_/ \\    Torch: 2.5.1+cu124. CUDA: 7.5. CUDA Toolkit: 12.4. Triton: 3.1.0\n",
            "\\        /    Bfloat16 = FALSE. FA [Xformers = 0.0.29.post1. FA2 = False]\n",
            " \"-____-\"     Free Apache license: http://github.com/unslothai/unsloth\n",
            "Unsloth: Fast downloading is enabled - ignore downloading bars which are red colored!\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4afe538bb09d4cf698235e135640ab6e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/5.96G [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "82860a16b8d944ff86f854f215535379",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "generation_config.json:   0%|          | 0.00/236 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ff5beb41522f4b80bdb35c436f478bb6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/52.9k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e07fdea2d68c4284b43df47b7f94fceb",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "special_tokens_map.json:   0%|          | 0.00/483 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cdc489ca63484d4abb3ac39a3f93e33c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/17.2M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "model, tokenizer = FastLanguageModel.from_pretrained(\n",
        "    model_name = \"unsloth/DeepSeek-R1-Distill-Llama-8B\",\n",
        "    max_seq_length = max_seq_length,\n",
        "    dtype = dtype,\n",
        "    load_in_4bit = load_in_4bit,\n",
        "    token = hf_token,\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "editable": false,
        "id": "DHTrWVzp6b_x"
      },
      "source": [
        "## Model inference before fine-tuning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T06:44:38.171365Z",
          "iopub.status.busy": "2025-01-25T06:44:38.171012Z",
          "iopub.status.idle": "2025-01-25T06:44:43.505327Z",
          "shell.execute_reply": "2025-01-25T06:44:43.504467Z",
          "shell.execute_reply.started": "2025-01-25T06:44:38.171335Z"
        },
        "id": "y36oW6mo6b_y",
        "papermill": {
          "duration": 0.014436,
          "end_time": "2024-09-25T13:13:09.483925",
          "exception": false,
          "start_time": "2024-09-25T13:13:09.469489",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [],
      "source": [
        "prompt_style = \"\"\"Below is an instruction that describes a task, paired with an input that provides further context.\n",
        "Write a response that appropriately completes the request.\n",
        "Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n",
        "\n",
        "### Instruction:\n",
        "You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning.\n",
        "Please answer the following medical question.\n",
        "\n",
        "### Question:\n",
        "{}\n",
        "\n",
        "### Response:\n",
        "<think>{}\"\"\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T06:44:52.901344Z",
          "iopub.status.busy": "2025-01-25T06:44:52.90099Z",
          "iopub.status.idle": "2025-01-25T06:45:35.406585Z",
          "shell.execute_reply": "2025-01-25T06:45:35.405815Z",
          "shell.execute_reply.started": "2025-01-25T06:44:52.901316Z"
        },
        "id": "Nt7aM7Rn6b_y",
        "outputId": "1e36e085-5394-4e6b-850e-46a0d6d74ae8",
        "trusted": true
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "<think>\n",
            "Okay, so I need to figure out what cystometry would show for this 61-year-old woman. Let's break it down step by step.\n",
            "\n",
            "First, the patient has a history of involuntary urine loss during activities like coughing or sneezing, but she doesn't leak at night. That makes me think about possible causes. Involuntary leakage during those activities is often related to urethral sphincter dysfunction, maybe due to intrinsic sphincter deficiency (ISD). So, the main issues here are likely sphincter weakness and perhaps some bladder capacity or contractility problems.\n",
            "\n",
            "She undergoes a gynecological exam and Q-tip test. I'm a bit rusty on the Q-tip test, but from what I remember, the Q-tip is a small catheter used to measure urethral resistance. It's inserted into the urethra, and the patient is asked to cough. If the resistance increases (indicating sphincter contraction), that's a positive sign. If it doesn't, it suggests sphincter deficiency.\n",
            "\n",
            "So, if the Q-tip test shows that the urethral resistance doesn't increase on coughing, that would support a diagnosis of ISD. That makes sense because the sphincter isn't contracting properly to prevent leakage.\n",
            "\n",
            "Now, thinking about cystometry. Cystometry is a test that assesses bladder function and sphincter dynamics. It involves filling the bladder and measuring how much it holds (residual volume) and the pressure it generates during contraction (detrusor pressure). The test also looks at how the sphincter responds to filling.\n",
            "\n",
            "In this case, the patient's history suggests she might have reduced bladder capacity because she doesn't leak at night, which is more common with underactive bladders. So, during cystometry, they'd likely find that the bladder doesn't hold much urine before needing to contract. The residual volume might be low because she doesn't leak at night, indicating her bladder doesn't usually hold much.\n",
            "\n",
            "As for detrusor contractions, the detrusor is the muscle layer of the bladder. If the detrusor isn't contracting well, it can't generate enough pressure to expel urine, leading to retention. However, in this scenario, since she's experiencing leakage during activities, it's more about the sphincter not closing, not the bladder's ability to contract. So, the detrusor contractions might be normal or even hyperactive, but the main issue is the sphincter deficiency.\n",
            "\n",
            "Putting it all together, the cystometry would likely show a low residual volume (since she doesn't retain urine at night) and normal detrusor contractions, but the primary issue is sphincter function. Therefore, the most relevant findings would be related to the sphincter's inability to prevent leakage.\n",
            "</think>\n",
            "\n",
            "Cystometry for this 61-year-old woman would most likely reveal a **low residual volume** and **normal detrusor contractions**. The primary finding would be related to sphincter function, indicating **intrinsic sphincter deficiency** given the positive Q-tip test results and her history of involuntary leakage during activities like coughing. The bladder's ability to contract appears normal, but the sphincter's inability to prevent leakage is the key issue.<｜end▁of▁sentence｜>\n"
          ]
        }
      ],
      "source": [
        "question = \"A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?\"\n",
        "\n",
        "\n",
        "FastLanguageModel.for_inference(model)  # Unsloth has 2x faster inference!\n",
        "inputs = tokenizer([prompt_style.format(question, \"\")], return_tensors=\"pt\").to(\"cuda\")\n",
        "\n",
        "outputs = model.generate(\n",
        "    input_ids=inputs.input_ids,\n",
        "    attention_mask=inputs.attention_mask,\n",
        "    max_new_tokens=1200,\n",
        "    use_cache=True,\n",
        ")\n",
        "response = tokenizer.batch_decode(outputs)\n",
        "print(response[0].split(\"### Response:\")[1])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T06:46:30.282007Z",
          "iopub.status.busy": "2025-01-25T06:46:30.281678Z",
          "iopub.status.idle": "2025-01-25T06:46:41.398923Z",
          "shell.execute_reply": "2025-01-25T06:46:41.398292Z",
          "shell.execute_reply.started": "2025-01-25T06:46:30.281986Z"
        },
        "id": "PLzmU3OH6b_y",
        "outputId": "57e968e5-f7d8-4140-8fa1-05cf5409f219",
        "papermill": {
          "duration": 5.943269,
          "end_time": "2024-09-25T13:13:09.461941",
          "exception": false,
          "start_time": "2024-09-25T13:13:03.518672",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Unsloth 2025.1.8 patched 32 layers with 32 QKV layers, 32 O layers and 32 MLP layers.\n"
          ]
        }
      ],
      "source": [
        "model = FastLanguageModel.get_peft_model(\n",
        "    model,\n",
        "    r=16,\n",
        "    target_modules=[\n",
        "        \"q_proj\",\n",
        "        \"k_proj\",\n",
        "        \"v_proj\",\n",
        "        \"o_proj\",\n",
        "        \"gate_proj\",\n",
        "        \"up_proj\",\n",
        "        \"down_proj\",\n",
        "    ],\n",
        "    lora_alpha=16,\n",
        "    lora_dropout=0,\n",
        "    bias=\"none\",\n",
        "    use_gradient_checkpointing=\"unsloth\",  # True or \"unsloth\" for very long context\n",
        "    random_state=3407,\n",
        "    use_rslora=False,\n",
        "    loftq_config=None,\n",
        ")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "editable": false,
        "id": "x_trNlci6b_y"
      },
      "source": [
        "## Loading and processing the dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T06:52:45.645903Z",
          "iopub.status.busy": "2025-01-25T06:52:45.6456Z",
          "iopub.status.idle": "2025-01-25T06:52:51.002333Z",
          "shell.execute_reply": "2025-01-25T06:52:51.001499Z",
          "shell.execute_reply.started": "2025-01-25T06:52:45.64588Z"
        },
        "id": "CwCzOgAy6b_y",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "train_prompt_style = \"\"\"Below is an instruction that describes a task, paired with an input that provides further context.\n",
        "Write a response that appropriately completes the request.\n",
        "Before answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\n",
        "\n",
        "### Instruction:\n",
        "You are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning.\n",
        "Please answer the following medical question.\n",
        "\n",
        "### Question:\n",
        "{}\n",
        "\n",
        "### Response:\n",
        "<think>\n",
        "{}\n",
        "</think>\n",
        "{}\"\"\"\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T06:52:51.003648Z",
          "iopub.status.busy": "2025-01-25T06:52:51.00341Z",
          "iopub.status.idle": "2025-01-25T06:52:56.433342Z",
          "shell.execute_reply": "2025-01-25T06:52:56.43268Z",
          "shell.execute_reply.started": "2025-01-25T06:52:51.003628Z"
        },
        "id": "Fo0OGx8w6b_z",
        "papermill": {
          "duration": 0.015203,
          "end_time": "2024-09-25T13:13:09.506291",
          "exception": false,
          "start_time": "2024-09-25T13:13:09.491088",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [],
      "source": [
        "EOS_TOKEN = tokenizer.eos_token  # Must add EOS_TOKEN\n",
        "\n",
        "\n",
        "def formatting_prompts_func(examples):\n",
        "    inputs = examples[\"Question\"]\n",
        "    cots = examples[\"Complex_CoT\"]\n",
        "    outputs = examples[\"Response\"]\n",
        "    texts = []\n",
        "    for input, cot, output in zip(inputs, cots, outputs):\n",
        "        text = train_prompt_style.format(input, cot, output) + EOS_TOKEN\n",
        "        texts.append(text)\n",
        "    return {\n",
        "        \"text\": texts,\n",
        "    }\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T06:53:30.006828Z",
          "iopub.status.busy": "2025-01-25T06:53:30.006539Z",
          "iopub.status.idle": "2025-01-25T06:53:40.189973Z",
          "shell.execute_reply": "2025-01-25T06:53:40.189098Z",
          "shell.execute_reply.started": "2025-01-25T06:53:30.006806Z"
        },
        "id": "V48w-68_6b_z",
        "papermill": {
          "duration": 13.8704,
          "end_time": "2024-09-25T13:13:23.383686",
          "exception": false,
          "start_time": "2024-09-25T13:13:09.513286",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [],
      "source": [
        "from datasets import load_dataset\n",
        "dataset = load_dataset(\"FreedomIntelligence/medical-o1-reasoning-SFT\",\"en\", split = \"train[0:500]\",trust_remote_code=True)\n",
        "dataset = dataset.map(formatting_prompts_func, batched = True,)\n",
        "dataset[\"text\"][0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        },
        "id": "kSCrcJxlAfoa",
        "outputId": "9d6ddebb-e390-4035-f3aa-977486080305"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "\"Below is an instruction that describes a task, paired with an input that provides further context. \\nWrite a response that appropriately completes the request. \\nBefore answering, think carefully about the question and create a step-by-step chain of thoughts to ensure a logical and accurate response.\\n\\n### Instruction:\\nYou are a medical expert with advanced knowledge in clinical reasoning, diagnostics, and treatment planning. \\nPlease answer the following medical question. \\n\\n### Question:\\nA 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?\\n\\n### Response:\\n<think>\\nOkay, let's think about this step by step. There's a 61-year-old woman here who's been dealing with involuntary urine leakages whenever she's doing something that ups her abdominal pressure like coughing or sneezing. This sounds a lot like stress urinary incontinence to me. Now, it's interesting that she doesn't have any issues at night; she isn't experiencing leakage while sleeping. This likely means her bladder's ability to hold urine is fine when she isn't under physical stress. Hmm, that's a clue that we're dealing with something related to pressure rather than a bladder muscle problem. \\n\\nThe fact that she underwent a Q-tip test is intriguing too. This test is usually done to assess urethral mobility. In stress incontinence, a Q-tip might move significantly, showing urethral hypermobility. This kind of movement often means there's a weakness in the support structures that should help keep the urethra closed during increases in abdominal pressure. So, that's aligning well with stress incontinence.\\n\\nNow, let's think about what would happen during cystometry. Since stress incontinence isn't usually about sudden bladder contractions, I wouldn't expect to see involuntary detrusor contractions during this test. Her bladder isn't spasming or anything; it's more about the support structure failing under stress. Plus, she likely empties her bladder completely because stress incontinence doesn't typically involve incomplete emptying. So, her residual volume should be pretty normal. \\n\\nAll in all, it seems like if they do a cystometry on her, it will likely show a normal residual volume and no involuntary contractions. Yup, I think that makes sense given her symptoms and the typical presentations of stress urinary incontinence.\\n</think>\\nCystometry in this case of stress urinary incontinence would most likely reveal a normal post-void residual volume, as stress incontinence typically does not involve issues with bladder emptying. Additionally, since stress urinary incontinence is primarily related to physical exertion and not an overactive bladder, you would not expect to see any involuntary detrusor contractions during the test.<｜end▁of▁sentence｜>\""
            ]
          },
          "execution_count": 22,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\"FreedomIntelligence/medical-o1-reasoning-SFT\", \"en\",split = \"train[0:500]\")\n",
        "dataset = dataset.map(formatting_prompts_func, batched = True,)\n",
        "dataset[\"text\"][0]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "editable": false,
        "id": "uFmyf5_w6b_z"
      },
      "source": [
        "## Setting up the model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "fc3993b236b24cbcaecc4e7192fe3254",
            "af30810fd0544ba4b904de2ce3f02cec",
            "1969437f6b7b4f92bd71058a0bd5755a",
            "83c6aa3318cf4c8c8be3fd4f8e9ba387",
            "a60b14d5d28547cea890f3c17ab2ac8a",
            "d02a1ccb5a914647a3e9464031d38b79",
            "5927fd687bc4479eae3bf5985445cc9c",
            "e5fd64b00bbb4e299930464604518bac",
            "3e6b9d78fce94a20bf09b22b12a58119",
            "4fcaab874f16414391964cd95ebaa648",
            "8c773d3f8c3842a887c37cf14ffcabee"
          ]
        },
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T06:56:57.951657Z",
          "iopub.status.busy": "2025-01-25T06:56:57.951347Z",
          "iopub.status.idle": "2025-01-25T06:57:03.526786Z",
          "shell.execute_reply": "2025-01-25T06:57:03.525951Z",
          "shell.execute_reply.started": "2025-01-25T06:56:57.951632Z"
        },
        "id": "9AuTcgxB6b_z",
        "outputId": "4ed3992b-e42f-4400-edc9-51660f4b0eb8",
        "papermill": {
          "duration": 1.901541,
          "end_time": "2024-09-25T13:13:25.296367",
          "exception": false,
          "start_time": "2024-09-25T13:13:23.394826",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fc3993b236b24cbcaecc4e7192fe3254",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Map (num_proc=2):   0%|          | 0/500 [00:00<?, ? examples/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "from trl import SFTTrainer\n",
        "from transformers import TrainingArguments\n",
        "from unsloth import is_bfloat16_supported\n",
        "\n",
        "trainer = SFTTrainer(\n",
        "    model=model,\n",
        "    tokenizer=tokenizer,\n",
        "    train_dataset=dataset,\n",
        "    dataset_text_field=\"text\",\n",
        "    max_seq_length=max_seq_length,\n",
        "    dataset_num_proc=2,\n",
        "    args=TrainingArguments(\n",
        "        per_device_train_batch_size=2,\n",
        "        gradient_accumulation_steps=4,\n",
        "        # Use num_train_epochs = 1, warmup_ratio for full training runs!\n",
        "        warmup_steps=5,\n",
        "        max_steps=60,\n",
        "        learning_rate=2e-4,\n",
        "        fp16=not is_bfloat16_supported(),\n",
        "        bf16=is_bfloat16_supported(),\n",
        "        logging_steps=10,\n",
        "        optim=\"adamw_8bit\",\n",
        "        weight_decay=0.01,\n",
        "        lr_scheduler_type=\"linear\",\n",
        "        seed=3407,\n",
        "        output_dir=\"outputs\",\n",
        "    ),\n",
        ")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "editable": false,
        "id": "QosSqAta6b_z"
      },
      "source": [
        "## Model training"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 638
        },
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T06:57:07.350469Z",
          "iopub.status.busy": "2025-01-25T06:57:07.350133Z",
          "iopub.status.idle": "2025-01-25T07:42:07.692916Z",
          "shell.execute_reply": "2025-01-25T07:42:07.692244Z",
          "shell.execute_reply.started": "2025-01-25T06:57:07.350441Z"
        },
        "id": "dGYprK9F6b_z",
        "outputId": "05b06c14-bfbb-4834-bf6f-80d66f91aff0",
        "papermill": {
          "duration": 1210.862521,
          "end_time": "2024-09-25T13:33:36.207919",
          "exception": false,
          "start_time": "2024-09-25T13:13:25.345398",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "==((====))==  Unsloth - 2x faster free finetuning | Num GPUs = 1\n",
            "   \\\\   /|    Num examples = 500 | Num Epochs = 1\n",
            "O^O/ \\_/ \\    Batch size per device = 2 | Gradient Accumulation steps = 4\n",
            "\\        /    Total batch size = 8 | Total steps = 60\n",
            " \"-____-\"     Number of trainable parameters = 41,943,040\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n"
          ]
        },
        {
          "data": {
            "application/javascript": "\n        window._wandbApiKey = new Promise((resolve, reject) => {\n            function loadScript(url) {\n            return new Promise(function(resolve, reject) {\n                let newScript = document.createElement(\"script\");\n                newScript.onerror = reject;\n                newScript.onload = resolve;\n                document.body.appendChild(newScript);\n                newScript.src = url;\n            });\n            }\n            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n            const iframe = document.createElement('iframe')\n            iframe.style.cssText = \"width:0;height:0;border:none\"\n            document.body.appendChild(iframe)\n            const handshake = new Postmate({\n                container: iframe,\n                url: 'https://wandb.ai/authorize'\n            });\n            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n            handshake.then(function(child) {\n                child.on('authorize', data => {\n                    clearTimeout(timeout)\n                    resolve(data)\n                });\n            });\n            })\n        });\n    ",
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
            "wandb: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " ··········\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m If you're specifying your api key in code, ensure this code is not shared publicly.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Consider setting the WANDB_API_KEY environment variable, or running `wandb login` from the command line.\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mnongbin\u001b[0m (\u001b[33mnongbin-tfswufe\u001b[0m) to \u001b[32mhttps://api.wandb.ai\u001b[0m. Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend.  Please refer to https://wandb.me/wandb-core for more information.\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "Tracking run with wandb version 0.19.5"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "Run data is saved locally in <code>/content/wandb/run-20250203_135029-0dv6lmzh</code>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "Syncing run <strong><a href='https://wandb.ai/nongbin-tfswufe/huggingface/runs/0dv6lmzh' target=\"_blank\">outputs</a></strong> to <a href='https://wandb.ai/nongbin-tfswufe/huggingface' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/developer-guide' target=\"_blank\">docs</a>)<br>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              " View project at <a href='https://wandb.ai/nongbin-tfswufe/huggingface' target=\"_blank\">https://wandb.ai/nongbin-tfswufe/huggingface</a>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              " View run at <a href='https://wandb.ai/nongbin-tfswufe/huggingface/runs/0dv6lmzh' target=\"_blank\">https://wandb.ai/nongbin-tfswufe/huggingface/runs/0dv6lmzh</a>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='60' max='60' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [60/60 19:55, Epoch 0/1]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Step</th>\n",
              "      <th>Training Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>10</td>\n",
              "      <td>1.918800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>20</td>\n",
              "      <td>1.461500</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>30</td>\n",
              "      <td>1.402300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>40</td>\n",
              "      <td>1.308800</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>50</td>\n",
              "      <td>1.344300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>60</td>\n",
              "      <td>1.314000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "trainer_stats = trainer.train()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "editable": false,
        "id": "3-wyYeic6b_z",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "# Save the fine-tuned model\n",
        "wandb.finish()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "editable": false,
        "id": "8yEZKVoo6b_z"
      },
      "source": [
        "## Model inference after fine-tuning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T07:42:07.694111Z",
          "iopub.status.busy": "2025-01-25T07:42:07.693911Z",
          "iopub.status.idle": "2025-01-25T07:42:52.981462Z",
          "shell.execute_reply": "2025-01-25T07:42:52.980824Z",
          "shell.execute_reply.started": "2025-01-25T07:42:07.694093Z"
        },
        "id": "9kVwqsga6b_0",
        "outputId": "32d74201-9815-4093-960c-912550b5ad97",
        "papermill": {
          "duration": 7.941358,
          "end_time": "2024-09-25T13:33:44.195877",
          "exception": false,
          "start_time": "2024-09-25T13:33:36.254519",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "<think>\n",
            "Okay, so let's think about this. We have a 61-year-old woman who's been dealing with involuntary urine loss during things like coughing or sneezing, but she's not leaking at night. That suggests she might have some kind of problem with her pelvic floor muscles or maybe her bladder.\n",
            "\n",
            "Now, she's got a gynecological exam and a Q-tip test. Let's break that down. The Q-tip test is usually used to check for urethral obstruction. If it's positive, that means there's something blocking the urethra, like a urethral stricture or something else.\n",
            "\n",
            "Given that she's had a positive Q-tip test, it's likely there's a urethral obstruction. That would mean her urethra is narrow, maybe due to a stricture or some kind of narrowing. So, her bladder can't empty properly during activities like coughing because the urethral obstruction is making it hard.\n",
            "\n",
            "Now, let's think about what happens when her bladder can't empty. If there's a urethral obstruction, the bladder is forced to hold more urine, increasing the residual volume. That's because her bladder doesn't empty completely. So, her residual volume is probably increased.\n",
            "\n",
            "Also, if her bladder can't empty properly, she might have increased detrusor contractions. These contractions are usually stronger to push the urine out. So, we expect her detrusor contractions to be increased.\n",
            "\n",
            "Putting it all together, if she has a urethral obstruction and a positive Q-tip test, we'd expect her cystometry results to show increased residual volume and increased detrusor contractions. That makes sense because of the obstruction and how her bladder is trying to compensate by contracting more.\n",
            "</think>\n",
            "Based on the findings of the gynecological exam and the positive Q-tip test, it is most likely that the cystometry would reveal increased residual volume and increased detrusor contractions. The positive Q-tip test indicates urethral obstruction, which would force the bladder to retain more urine, increasing the residual volume. Additionally, the detrusor contractions may be increased as the bladder compensates for the obstruction, attempting to push more urine out despite the obstruction.<｜end▁of▁sentence｜>\n"
          ]
        }
      ],
      "source": [
        "question = \"A 61-year-old woman with a long history of involuntary urine loss during activities like coughing or sneezing but no leakage at night undergoes a gynecological exam and Q-tip test. Based on these findings, what would cystometry most likely reveal about her residual volume and detrusor contractions?\"\n",
        "\n",
        "\n",
        "FastLanguageModel.for_inference(model)  # Unsloth has 2x faster inference!\n",
        "inputs = tokenizer([prompt_style.format(question, \"\")], return_tensors=\"pt\").to(\"cuda\")\n",
        "\n",
        "outputs = model.generate(\n",
        "    input_ids=inputs.input_ids,\n",
        "    attention_mask=inputs.attention_mask,\n",
        "    max_new_tokens=1200,\n",
        "    use_cache=True,\n",
        ")\n",
        "response = tokenizer.batch_decode(outputs)\n",
        "print(response[0].split(\"### Response:\")[1])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T07:53:20.408914Z",
          "iopub.status.busy": "2025-01-25T07:53:20.408711Z",
          "iopub.status.idle": "2025-01-25T07:54:02.523178Z",
          "shell.execute_reply": "2025-01-25T07:54:02.522504Z",
          "shell.execute_reply.started": "2025-01-25T07:53:20.408896Z"
        },
        "id": "TpM4uf7E6b_0",
        "outputId": "683efd00-33dc-46f6-833d-828c0fd275ea",
        "trusted": true
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n",
            "<think>\n",
            "Alright, let's think about this. We've got a 59-year-old guy who's got a fever, chills, night sweats, and feels pretty tired. That sounds like a classic picture of an infection, right? And there's a vegetation on his aortic valve. Hmm, that's not great news. \n",
            "\n",
            "Now, looking at the blood culture results: gram-positive, catalase-negative, gamma-hemolytic cocci in chains. That means it's a Streptococcus species, specifically Streptococcus pyogenes. It's not Streptococcus pneumoniae because it doesn't grow in NaCl. And this guy's symptoms, combined with the vegetation, point towards endocarditis, which is an infection of the heart valves.\n",
            "\n",
            "Okay, so let's figure out what could have caused this. I'm thinking about infections, not just from bacteria, but also from fungi or viruses. It's important to consider these possibilities because sometimes you can miss something. \n",
            "\n",
            "But wait, let's think about the main culprit. Streptococcus pyogenes is pretty aggressive and can cause a lot of problems, especially if it gets into the bloodstream. That can lead to endocarditis if it takes hold in the heart. Now, why would this happen?\n",
            "\n",
            "Well, there are a few things that can increase the risk. One big factor is the presence of an underlying condition that weakens the immune system. That could be something like diabetes, heart disease, or even something like chronic kidney disease. All these things can make it harder for the body to fight off infections effectively.\n",
            "\n",
            "Another thought is about medical procedures. Sometimes, when you have a medical procedure or device, like a pacemaker or a catheter, there's a risk of infection if it's not properly sterilized or if it's left in place for too long. This is called Nosocomial infection.\n",
            "\n",
            "Oh, and there's also the possibility of it being due to the use of steroids or other medications. These can suppress the immune system and make it easier for an infection to take hold. \n",
            "\n",
            "Hmm, let's think about these factors. We've got diabetes, heart disease, and medical procedures. These are all possible reasons why Streptococcus pyogenes could be causing this. \n",
            "\n",
            "But let's not forget that Streptococcus pyogenes itself can be quite virulent. It doesn't need a weakened immune system to cause an infection. It can just thrive in the body naturally. So, it's not just about the environment, but also about the nature of the bacteria itself. \n",
            "\n",
            "Alright, so to sum up, the most likely causes for this infection could be due to a weakened immune system, like from diabetes or heart disease, or it could be from medical procedures that aren't properly kept sterile, or even from using medications that suppress the immune system. \n",
            "\n",
            "In this case, given that it's a gram-positive, catalase-negative, gamma-hemolytic cocci in chains, it's definitely Streptococcus pyogenes. So, the most likely reason this happened is because of an underlying medical condition that made it harder for the body to fight off the infection. \n",
            "\n",
            "So, yeah, it's probably diabetes or heart disease that made this happen. That's my best guess!\n",
            "</think>\n",
            "The most likely predisposing factor for this patient's condition is an underlying medical condition that weakens the immune system, such as diabetes or heart disease. These conditions can make it harder for the body to combat infections effectively, allowing the infection to take hold and cause endocarditis due to Streptococcus pyogenes.<｜end▁of▁sentence｜>\n"
          ]
        }
      ],
      "source": [
        "question = \"A 59-year-old man presents with a fever, chills, night sweats, and generalized fatigue, and is found to have a 12 mm vegetation on the aortic valve. Blood cultures indicate gram-positive, catalase-negative, gamma-hemolytic cocci in chains that do not grow in a 6.5% NaCl medium. What is the most likely predisposing factor for this patient's condition?\"\n",
        "\n",
        "inputs = tokenizer([prompt_style.format(question, \"\")], return_tensors=\"pt\").to(\"cuda\")\n",
        "\n",
        "outputs = model.generate(\n",
        "    input_ids=inputs.input_ids,\n",
        "    attention_mask=inputs.attention_mask,\n",
        "    max_new_tokens=1200,\n",
        "    use_cache=True,\n",
        ")\n",
        "response = tokenizer.batch_decode(outputs)\n",
        "print(response[0].split(\"### Response:\")[1])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "editable": false,
        "id": "MbpJ1ZaY6b_0"
      },
      "source": [
        "## Saving the model locally"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T07:43:06.366389Z",
          "iopub.status.busy": "2025-01-25T07:43:06.366148Z",
          "iopub.status.idle": "2025-01-25T07:43:13.586931Z",
          "shell.execute_reply": "2025-01-25T07:43:13.585982Z",
          "shell.execute_reply.started": "2025-01-25T07:43:06.366367Z"
        },
        "id": "jazgZ1sx6b_0",
        "papermill": {
          "duration": 1.080093,
          "end_time": "2024-09-25T13:33:49.520495",
          "exception": false,
          "start_time": "2024-09-25T13:33:48.440402",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [],
      "source": [
        "new_model_online = \"kingabzpro/DeepSeek-R1-Medical-COT\"\n",
        "new_model_local = \"DeepSeek-R1-Medical-COT\"\n",
        "model.save_pretrained(new_model_local) # Local saving\n",
        "tokenizer.save_pretrained(new_model_local)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "editable": false,
        "id": "tju4gSOp6b_0"
      },
      "source": [
        "## Pushing the model to Hugging Face hub"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T07:43:13.588028Z",
          "iopub.status.busy": "2025-01-25T07:43:13.587755Z",
          "iopub.status.idle": "2025-01-25T07:43:32.143116Z",
          "shell.execute_reply": "2025-01-25T07:43:32.142247Z",
          "shell.execute_reply.started": "2025-01-25T07:43:13.587997Z"
        },
        "id": "5I9egNBA6b_0",
        "papermill": {
          "duration": 14.531527,
          "end_time": "2024-09-25T13:34:04.065269",
          "exception": false,
          "start_time": "2024-09-25T13:33:49.533742",
          "status": "completed"
        },
        "tags": [],
        "trusted": true
      },
      "outputs": [],
      "source": [
        "model.push_to_hub(new_model_online) # Online saving\n",
        "tokenizer.push_to_hub(new_model_online) # Online saving"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "editable": false,
        "execution": {
          "iopub.execute_input": "2025-01-25T07:45:56.087597Z",
          "iopub.status.busy": "2025-01-25T07:45:56.087215Z",
          "iopub.status.idle": "2025-01-25T07:53:20.405882Z",
          "shell.execute_reply": "2025-01-25T07:53:20.405181Z",
          "shell.execute_reply.started": "2025-01-25T07:45:56.087571Z"
        },
        "id": "JkY85SIc6b_0",
        "trusted": true
      },
      "outputs": [],
      "source": [
        "model.save_pretrained_merged(new_model_local, tokenizer, save_method = \"merged_16bit\",)\n",
        "model.push_to_hub_merged(new_model_online, tokenizer, save_method = \"merged_16bit\")"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kaggle": {
      "accelerator": "nvidiaTeslaT4",
      "dataSources": [],
      "dockerImageVersionId": 30839,
      "isGpuEnabled": true,
      "isInternetEnabled": false,
      "language": "python",
      "sourceType": "notebook"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.12"
    },
    "papermill": {
      "default_parameters": {},
      "duration": 1500.86091,
      "end_time": "2024-09-25T13:34:07.208125",
      "environment_variables": {},
      "exception": null,
      "input_path": "__notebook__.ipynb",
      "output_path": "__notebook__.ipynb",
      "parameters": {},
      "start_time": "2024-09-25T13:09:06.347215",
      "version": "2.6.0"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "029a61b64a0844dab51a9eb8ab949bf0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4e17d9034b6d41c58bdc8ca94ea0c2fd",
            "placeholder": "​",
            "style": "IPY_MODEL_ac02a01ce16e4fbca4ff1819a5153088",
            "value": " 52.9k/52.9k [00:00&lt;00:00, 4.40MB/s]"
          }
        },
        "045a773300914b91a43f7f4922baf8cc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "08749a83ae4a47128e23b69dd791ab6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0a389bfe361f43578d55001f88a92fea": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0f3a15d30fca4e5aa6321f1497e9c449": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1969437f6b7b4f92bd71058a0bd5755a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e5fd64b00bbb4e299930464604518bac",
            "max": 500,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3e6b9d78fce94a20bf09b22b12a58119",
            "value": 500
          }
        },
        "1992bceec6b54ac2827285d39c8c12ee": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1cd217e8233d495db151f5debdf9f00b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_eb8becd866564ffd859e22bbc61a93e8",
            "max": 17209530,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7a50ddf657a04aef8c68c4100323fcea",
            "value": 17209530
          }
        },
        "1f2e54775e6e4288a2a772b75d104fbb": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1fe6b27ff0cc4e9fa39b0b5840a25aec": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1992bceec6b54ac2827285d39c8c12ee",
            "placeholder": "​",
            "style": "IPY_MODEL_45f8a2d59bfe46008feebab8981d7146",
            "value": "model.safetensors: 100%"
          }
        },
        "2e82c8498b854d41842977ef8f9bbb66": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "30b0da59ccb241439ee988cf52bf81b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1f2e54775e6e4288a2a772b75d104fbb",
            "placeholder": "​",
            "style": "IPY_MODEL_e15cb3d0d72a475ea39eb1ef761dc827",
            "value": "tokenizer.json: 100%"
          }
        },
        "33edc00314bf444188376c01ea025038": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e68571618bf748ebbf6d3e244e0e7f29",
            "placeholder": "​",
            "style": "IPY_MODEL_e1d763ec083b49d0900cfe5d42d9b214",
            "value": "generation_config.json: 100%"
          }
        },
        "3914a21f18114d6784bbb8f4867be344": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3e6b9d78fce94a20bf09b22b12a58119": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "41c36bc4fcb542a0bbc3443cd4c742ea": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "424e62fec0c0416ba13f5863391e08f9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "44d3cf803d244775826d566c4d9ca859": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "45f8a2d59bfe46008feebab8981d7146": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4afe538bb09d4cf698235e135640ab6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1fe6b27ff0cc4e9fa39b0b5840a25aec",
              "IPY_MODEL_f811d4e961914901b3b458934512eb86",
              "IPY_MODEL_9812fb88a8b3411d8675d238161dd7aa"
            ],
            "layout": "IPY_MODEL_6dbb94366cc941a8ba336ba61ebf99ba"
          }
        },
        "4e17d9034b6d41c58bdc8ca94ea0c2fd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4e3945399c1642c1bdf724f8a01fcd02": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_972543b62d0a4d908101b7abc8c1ca15",
            "max": 236,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8e5e7c77e88a4d3b8b606f240b627a9d",
            "value": 236
          }
        },
        "4fcaab874f16414391964cd95ebaa648": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "51ad33f2e8dc44f79a8e8fd9a86e22b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5927fd687bc4479eae3bf5985445cc9c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6dbb94366cc941a8ba336ba61ebf99ba": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "724109739173448fb1965e01895f51a2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "76bc4ff5adb746a69954699cf5465c2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_045a773300914b91a43f7f4922baf8cc",
            "placeholder": "​",
            "style": "IPY_MODEL_9ef2064efb05485b8b63bb3db815a8ef",
            "value": " 17.2M/17.2M [00:00&lt;00:00, 42.1MB/s]"
          }
        },
        "7a50ddf657a04aef8c68c4100323fcea": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7ae90e66d3f1459ab5e0c202cf4228ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e5784f34cf184023ac60d15e4c347688",
            "placeholder": "​",
            "style": "IPY_MODEL_3914a21f18114d6784bbb8f4867be344",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "80b57123cbb64b2c9a27c5159a104556": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "82860a16b8d944ff86f854f215535379": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_33edc00314bf444188376c01ea025038",
              "IPY_MODEL_4e3945399c1642c1bdf724f8a01fcd02",
              "IPY_MODEL_f58d5860eb854c698f6558e4422acced"
            ],
            "layout": "IPY_MODEL_0f3a15d30fca4e5aa6321f1497e9c449"
          }
        },
        "83c6aa3318cf4c8c8be3fd4f8e9ba387": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4fcaab874f16414391964cd95ebaa648",
            "placeholder": "​",
            "style": "IPY_MODEL_8c773d3f8c3842a887c37cf14ffcabee",
            "value": " 500/500 [00:02&lt;00:00, 99.67 examples/s]"
          }
        },
        "8c773d3f8c3842a887c37cf14ffcabee": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8e5e7c77e88a4d3b8b606f240b627a9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "972543b62d0a4d908101b7abc8c1ca15": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9812fb88a8b3411d8675d238161dd7aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_424e62fec0c0416ba13f5863391e08f9",
            "placeholder": "​",
            "style": "IPY_MODEL_0a389bfe361f43578d55001f88a92fea",
            "value": " 5.96G/5.96G [01:47&lt;00:00, 93.1MB/s]"
          }
        },
        "9ab8e04684054331928edd757ce8dea3": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9ef2064efb05485b8b63bb3db815a8ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a60b14d5d28547cea890f3c17ab2ac8a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ac02a01ce16e4fbca4ff1819a5153088": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "adb31ccfc7ef415fb47a40ede28fe9db": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "af30810fd0544ba4b904de2ce3f02cec": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d02a1ccb5a914647a3e9464031d38b79",
            "placeholder": "​",
            "style": "IPY_MODEL_5927fd687bc4479eae3bf5985445cc9c",
            "value": "Map (num_proc=2): 100%"
          }
        },
        "b01764e9ac74437aa174b132789417af": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_80b57123cbb64b2c9a27c5159a104556",
            "max": 52946,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cf8df2dc586a4a9bba21066c50805fda",
            "value": 52946
          }
        },
        "b0dc37d789084b8e95ae90c155699ab7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_41c36bc4fcb542a0bbc3443cd4c742ea",
            "placeholder": "​",
            "style": "IPY_MODEL_08749a83ae4a47128e23b69dd791ab6e",
            "value": "special_tokens_map.json: 100%"
          }
        },
        "b8428a11dcfe4a2ea6d47b6e010fe3d2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ba91815af2f84652987bd01420bb061f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bee5e75849cf448fa6c5eb2ef61f48d4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cdc489ca63484d4abb3ac39a3f93e33c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_30b0da59ccb241439ee988cf52bf81b3",
              "IPY_MODEL_1cd217e8233d495db151f5debdf9f00b",
              "IPY_MODEL_76bc4ff5adb746a69954699cf5465c2f"
            ],
            "layout": "IPY_MODEL_bee5e75849cf448fa6c5eb2ef61f48d4"
          }
        },
        "cf82cef940944506bbb654363a48cdb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ba91815af2f84652987bd01420bb061f",
            "max": 483,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f500319ad888458f902f5fc4fc30d528",
            "value": 483
          }
        },
        "cf8df2dc586a4a9bba21066c50805fda": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d02a1ccb5a914647a3e9464031d38b79": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dca1f2da331542ae85818b544827dd96": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e07fdea2d68c4284b43df47b7f94fceb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b0dc37d789084b8e95ae90c155699ab7",
              "IPY_MODEL_cf82cef940944506bbb654363a48cdb7",
              "IPY_MODEL_fe6b626df24943978a0af261d29c7ee4"
            ],
            "layout": "IPY_MODEL_724109739173448fb1965e01895f51a2"
          }
        },
        "e15cb3d0d72a475ea39eb1ef761dc827": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e1d763ec083b49d0900cfe5d42d9b214": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e5784f34cf184023ac60d15e4c347688": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e5fd64b00bbb4e299930464604518bac": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e68571618bf748ebbf6d3e244e0e7f29": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eb8becd866564ffd859e22bbc61a93e8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f500319ad888458f902f5fc4fc30d528": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f58d5860eb854c698f6558e4422acced": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b8428a11dcfe4a2ea6d47b6e010fe3d2",
            "placeholder": "​",
            "style": "IPY_MODEL_dca1f2da331542ae85818b544827dd96",
            "value": " 236/236 [00:00&lt;00:00, 17.9kB/s]"
          }
        },
        "f811d4e961914901b3b458934512eb86": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "danger",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2e82c8498b854d41842977ef8f9bbb66",
            "max": 5964186418,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_adb31ccfc7ef415fb47a40ede28fe9db",
            "value": 5964185850
          }
        },
        "fc3993b236b24cbcaecc4e7192fe3254": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_af30810fd0544ba4b904de2ce3f02cec",
              "IPY_MODEL_1969437f6b7b4f92bd71058a0bd5755a",
              "IPY_MODEL_83c6aa3318cf4c8c8be3fd4f8e9ba387"
            ],
            "layout": "IPY_MODEL_a60b14d5d28547cea890f3c17ab2ac8a"
          }
        },
        "fe6b626df24943978a0af261d29c7ee4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_44d3cf803d244775826d566c4d9ca859",
            "placeholder": "​",
            "style": "IPY_MODEL_51ad33f2e8dc44f79a8e8fd9a86e22b5",
            "value": " 483/483 [00:00&lt;00:00, 44.3kB/s]"
          }
        },
        "ff5beb41522f4b80bdb35c436f478bb6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7ae90e66d3f1459ab5e0c202cf4228ab",
              "IPY_MODEL_b01764e9ac74437aa174b132789417af",
              "IPY_MODEL_029a61b64a0844dab51a9eb8ab949bf0"
            ],
            "layout": "IPY_MODEL_9ab8e04684054331928edd757ce8dea3"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}
